Feed-forward NN with 256 ReLU neurons for each of 2 hidden layers.
In [1]:
import tensorflow as tf
import numpy as np
In [18]:
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
In [7]:
# each layer consists of a f(xW + b) where f is a ReLU activation function
# For ReLU units, a study published in 2015 by He et al. demonstrates that the variance
# of weights in a network should be 2/n_in , where n_in is the number inputs coming into in the neuron.
def layer(input, weight_shape, bias_shape):
weight_stddev = (2.0 / weight_shape[0])**0.5 # variance=2/n_in as explained above
w_init = tf.random_normal_initializer(stddev=weight_stddev)
bias_init = tf.constant_initializer(value=0)
W = tf.get_variable("W", weight_shape, initializer=w_init)
b = tf.get_variable("b", bias_shape, initializer=bias_init)
return tf.nn.relu(tf.matmul(input, W) + b)
In [8]:
# inference is the way we pass our input x through the nn of 2 hidden layers and 1 output layer
def inference(x):
with tf.variable_scope("hidden_1"):
hidden_1 = layer(x, [784, 256], [256])
with tf.variable_scope("hidden_2"):
hidden_2 = layer(hidden_1, [256, 256], [256])
with tf.variable_scope("output"):
output = layer(hidden_2, [256, 10], [10])
return output
In [14]:
# softmax moved form inference to loss function for performance improvement
def loss(output, y):
xentropy = tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=y)
loss = tf.reduce_mean(xentropy)
return loss
In [15]:
def training(cost, global_step):
tf.summary.tensor_summary("cost", cost)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.minimize(cost, global_step=global_step)
return train_op
In [19]:
def evaluate(output, y):
# compare indices of predicted class, if equal (correct classification) set 1 otherwise 0
correct_prediction = tf.equal(tf.argmax(output, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
return accuracy
In [20]:
# Parameters
learning_rate = 0.01
training_epochs = 100
batch_size = 100
display_step = 1
In [ ]:
from tqdm import tqdm
# program flow
with tf.Graph().as_default():
# mnist data image of shape 28*28=784
x = tf.placeholder("float", [None, 784])
# 0-9 digits recognition => 10 classes
y = tf.placeholder("float", [None, 10])
output = inference(x)
cost = loss(output, y)
global_step = tf.Variable(0, name='global_step', trainable=False)
train_op = training(cost, global_step)
eval_op = evaluate(output, y)
# tf.merge_all_summaries in order to collect all summary statistics
# use a tf.train.SummaryWriter to write the log to disk.
summary_op = tf.summary.merge_all()
saver = tf.train.Saver()
sess = tf.Session()
# write to tensorboard graph api
summary_writer = tf.summary.FileWriter(
"logistic_logs/", graph=sess.graph)
init_op = tf.global_variables_initializer()
sess.run(init_op)
# training cycle
for epoch in tqdm(range(training_epochs)):
avg_cost = 0.
total_batch = int(mnist.train.num_examples / batch_size)
# Loop over all batches
for i in range(total_batch):
mbatch_x, mbatch_y = mnist.train.next_batch(batch_size)
# Fit training using batch data
feed_dict = {x: mbatch_x, y: mbatch_y}
sess.run(train_op, feed_dict=feed_dict)
# Compute average loss
minibatch_cost = sess.run(cost, feed_dict=feed_dict)
avg_cost += minibatch_cost / total_batch
# Display logs per epoch step
if epoch % display_step == 0:
val_feed_dict = {
x: mnist.validation.images,
y: mnist.validation.labels
}
accuracy = sess.run(eval_op, feed_dict=val_feed_dict)
print("Validation Error in epoch %s: %.11f" % (epoch, 1 - accuracy))
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, sess.run(global_step))
saver.save(
sess,
"logistic_logs/model-checkpoint",
global_step=global_step)
test_feed_dict = {x: mnist.test.images, y: mnist.test.labels}
accuracy = sess.run(eval_op, feed_dict=test_feed_dict)
print("Test Accuracy:", accuracy)
In [ ]: